import requests
from bs4 import BeautifulSoup
import file_util as f
import json

liepin_h = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'acw_tc=276077c916557038244947398ecdb91aa6d89c2161a5c8692a080499391f49; __gc_id=f4b8b7438f5142949da8d7b9f6a8d8ea; __uuid=1655703825147.21; __tlog=1655703825294.13%7C00000000%7C00000000%7C00000000%7C00000000; Hm_lvt_a2647413544f5a04f00da7eee0d5e200=1655703825; __s_bid=e9b8c46489dde1d17d9d437cbf477b1d9385; UniqueKey=2b8b5b37e584d8211a667e00ad0d9493; lt_auth=v%2BkMO3xRxwr7sCTRjGUNt6dP29OvBGXIpnwOhkoB0tG6X%2FLg4P3jQwyOr7ADxAMhkUl2dcULN7X7Muj6wXNP6EEawGuviZ%2B3veW%2B02ECTeNcN8W2vezHl8zRQpQclUAC8nFbtkI%3D; access_system=C; fe_se=-1655704805536; inited_user=d2a3ee23b16667b9f4a7b63f0ddbc25f; user_roles=0; user_photo=5f8fa3baf6d1ab58476f324908u.png; user_name=%E4%BA%8E%E7%87%95; need_bind_tel=false; new_user=false; c_flag=0fb0f3cee7b8128102135b6a6673d1c6; imClientId=b181e60c0111c7562d1236b2021a1791; imId=b181e60c0111c756a8e1f303d5286b64; imClientId_0=b181e60c0111c7562d1236b2021a1791; imId_0=b181e60c0111c756a8e1f303d5286b64; imApp_0=1; fe_im_connectJson_0=%7B%220_2b8b5b37e584d8211a667e00ad0d9493%22%3A%7B%22socketConnect%22%3A%221%22%2C%22connectDomain%22%3A%22liepin.com%22%7D%7D; Hm_lpvt_a2647413544f5a04f00da7eee0d5e200=1655704935; __session_seq=14; __uv_seq=14; fe_im_socketSequence_new_0=7_5_1',
    'Host': 'www.liepin.com',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="102", "Microsoft Edge";v="102"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44'
}


def do_liepin():
    t = ['人力资源', '法务', '财务', '投资', '证券', '审计']
    for tt in t:
        for i in range(30):
            liepin_url = 'https://www.liepin.com/zhaopin/?headId=a9d8dac1cbda7058ebdcb51892ceb868&ckId=m079ba3ifjbz9i8mruiyrbz51qirerb8&oldCkId=4549caf8befaa0238a57f4fa583ba3fa&fkId=scyavlhcaehip9nlxj9ch4puq6hnnooa&skId=r969vhkj3eo4demdtdriwi5aenhs7j38&sfrom=search_job_pc&key='+tt+'&dq=060020&currentPage='+str(i)+'&scene=page'
            liepin_resp = requests.get(liepin_url, headers=liepin_h).content.decode('utf-8')
            # print(liepin_resp)
            liepin_html = BeautifulSoup(liepin_resp, 'html.parser')
            print(liepin_html.title)
            liepin_list = liepin_html.find_all('div', class_='job-list-item')
            print(len(liepin_list))
            # 公司名称	公司性质	所属行业	人数规模	部门	岗位名称	岗位级别	薪资信息
            for liepin in liepin_list:
                # print(liepin)
                comp_name = liepin.find_all('span', class_='company-name')[0].get_text()
                # comp_url
                comp_url = liepin.find_all('a', {'data-nick':"job-detail-job-info"})[0].get('href')
                tags = liepin.find_all('div', class_='company-tags-box')
                comp_industry = ""
                comp_guimo = ""
                if len(tags) > 0:
                    comp_tags = tags[0].select('span')
                    print(len(comp_tags))
                    comp_industry = comp_tags[0].get_text()
                    comp_guimo = comp_tags[len(comp_tags)-1].get_text()
                # 部门没有
                # 岗位名称
                comp_gangwei = liepin.find_all('div', class_='ellipsis-1')[0].get_text()
                # 岗位级别 没有
                # 薪资信息
                comp_xinzi = liepin.find_all('span', class_='job-salary')[0].get_text()
                f.write_to_new_file(tt+'.txt', comp_name + '\t' + comp_industry + '\t' + comp_guimo + '\t' + comp_gangwei + '\t' + comp_xinzi+'\t' + comp_url + '\n')

job_header = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Connection': 'keep-alive',
    'Cookie': '_uab_collina=165572550162126889795978; acw_tc=76b20ff016557255004547388e3ab17fb9aaaa7ef5d56d00d726e15306ff02; guid=1fdf0d482e8a75c76c23b14081c14e72; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_expansion%3D; search=jobarea%7E%60070200%7C%21ord_field%7E%600%7C%21recentSearch0%7E%60070200%A1%FB%A1%FA000000%A1%FB%A1%FA0400%2C3300%2C8300%2C0600%2C1100%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21; ssxmod_itna=eq0OqIxUOxkGCDzxAOe5=i=OPiIZTyT9WnTuPDsqOrDSxGKidDqxBWmlexDQr1l/0wG+WOBhBbYzLKRmiq4EzjYPTsoxCPGnDBK+tHixYYkDt4DTD34DYDixib1xi5GRD0KDF7dy/1yDYPDE0KDaxDbDiekIxGCDeKD0xuFDQKDu2FqY0qkoUawcagCYmBxjxG1H40HOj8vrRfLljH+pzGQ1ADDUDnUq0qNMzbqTY73Ix0koq0Oya5s1agUXc5z4/ueEnro/nxxrm23YW0qTGW2WCD73WC0rjDrqAdbGkhNFQDDircNHYD==; ssxmod_itna2=eq0OqIxUOxkGCDzxAOe5=i=OPiIZTyT9WnTqikANqDlccEDjbq+QkMnmCQ1U154RKDDIHTet0u0xO8rCYNe=8xApxRGt2BcjRRi685O1ge9KW4F5dFI4L7MfULlcm2t5k8Byp=v=z9cPK4+a5325WXcczQ/c+2haT6nGHQicQwEFrlBHdqQFG=ow=fc2tMAK47phvdlc=6G0qRm8t6tuqgGctX=27MAL7i6fIe=oWekPbmaoha=x6ac85dy8aaF1aYju9zU1C6IyQ6pv=pXCsVgazfkp==A0q4cGtC2IIEMZSMLsEF16VaPEotBe5k1P9WElYejZ=7iKhlvu=wln64t=9ZeRn3plKAnpNSU9WvQ46X1GY+bRa2c0IFzPF/ii4TpiI8z6P8iRG0K30LpNbKYSQWZGR/WmomYzAWi3E0eEhDaXCfp/jA9UK=K6tRAza=SAj3cpUPWDG2lBeA4v+hililK+4WG8VmhWGkMDCoDxD08DiQ+YD===',
    'Host': 'search.51job.com',
    'Referer': 'https://search.51job.com/list/070200,000000,0400%252c3300%252c8300%252c0600%252c1100,00,9,99,+,2,2.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=',
    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'Sec-Fetch-Dest': 'empty',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Site': 'same-origin',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest'
}

def do_51job():
    # t = ['人力资源','律师/法务/合规','财务/审计/税务','金融/证券/期货','投融资']
    # for tt in t:
    for i in range(1, 134):
        url = 'https://search.51job.com/list/070200,000000,0400%252c3300%252c8300%252c0600%252c1100,00,9,99,+,2,'+str(i)+'.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
        comp_html = requests.get(url, headers=job_header).content.decode('gbk')
        comp = json.loads(comp_html)
        jds = comp.get('engine_jds')
        # 企业名称 公司性质 所属行业 人数规模 部门 岗位名称 岗位级别 薪资信息
        for jd in jds:
            comp_name = jd.get("company_name")
            comp_type = jd.get("companytype_text")
            comp_hangye = jd.get("companyind_text")
            comp_guimo = jd.get("companysize_text")
            comp_job = jd.get("job_name")
            salary = jd.get("providesalary_text")
            date_text = jd.get("issuedate")
            f.write_to_new_file('job51.txt', comp_name + '\t' + comp_type + '\t' + comp_hangye + '\t' + comp_guimo + '\t' + comp_job + '\t' + salary + '\t' + date_text + "\n")

if __name__ == '__main__':
    # do_liepin()
    do_51job()
    print('finish!')
